# -*- coding: utf-8 -*-
'''
Created on 2017年3月20日

@author: ZhuJiahui
'''

import os
import time
import numpy as np
from file_utils.file_writer import quick_write_1d_to_text
from global_info.global_nlp import GlobalNLP

def random_mixture(read_directory, write_filename):
    file_list = os.listdir(read_directory)
    
    all_text = []
    all_count = 0
    for i in range(len(file_list)):
        with open(read_directory + '/' + file_list[i], 'r') as f:
            
            file_str = file_list[i].replace(".txt", "")
            for eachline in f:
                all_text.append(file_str + GlobalNLP.EN_COLUMN_DELIMITER + eachline.strip())
                all_count += 1

    shuffle_index = np.arange(all_count)
    np.random.shuffle(shuffle_index)
    to_write = ["EMPTY" for j in range(all_count)]
    
    for i in range(all_count):
        to_write[shuffle_index[i]] = all_text[i]
    
    quick_write_1d_to_text(write_filename, to_write)
        

if __name__ == '__main__':
    start = time.clock()    
    now_directory = os.getcwd()
    root_directory = os.path.dirname(now_directory) + '/'
    read_directory1 = root_directory + u'dataset/20newsgroup/train_text'
    write_filename1 = root_directory + u'dataset/20newsgroup/mixture_train_text.txt'
    read_directory2 = root_directory + u'dataset/20newsgroup/test_text'
    write_filename2 = root_directory + u'dataset/20newsgroup/mixture_test_text.txt'
    
    random_mixture(read_directory1, write_filename1)
    random_mixture(read_directory2, write_filename2)
    
    print('Total time %f seconds' % (time.clock() - start))
    